<?php
/**
 * Created by PhpStorm.
 * User: lih
 * Date: 2018/5/11
 * Time: 23:03
 */

namespace console\models\crawl;

use Yii;
use yii\db\Query;
use common\components\CrawlHelper;
use common\models\ForumPost;
use common\models\PostMedia;

/**
 *
 * @property CrawlHelper $crawlHelper
 */
class CrawlTask
{
    const RULE_PATH_PREFIX = "console/assets/crawl";

    const MAX_LIST_PAGE = 5;

    public $crawlHelper;

    /**
     * @param $url string
     * @param $taskAction string
     */
    public function startCrawlList($url, $taskAction)
    {
        $taskProperties = $this->getTaskPropertiesByAction($taskAction);
        if(empty($taskProperties)) {
            return;
        }
        $crawlStat = array(
            'url' => $url,
            'starttime' => time(),
            'endtime' => time(),
            'listcount' => 0,
            'availablecount' => 0,
            'errordetails' => array(),
        );

        $crawlHelper = $this->getCrawlHelper();
        $listResult = $crawlHelper->loadList($url);
        $jobCount = 0;
        if(!empty($listResult)) {
            $crawlStat['listcount'] = count($listResult);
            $this->sleepAlittle();
            for($i = 0; $i < count($listResult); $i++) {
                $jobCount++;
//                if($jobCount > 3) {
//                    break;
//                }
                $listItem = $listResult[$i];
                if(!empty($listItem['_detailUrl'])) {
                    $detailUrl = $listItem['_detailUrl'];
                    unset($listItem['_detailUrl']);
                    if($this->isUrlVisited($detailUrl)) {
                        continue;
                    }
                    $detailResult = $this->doCrawlDetail($detailUrl);
                    if(!empty($detailResult)) {
                        $crawlContentArray = [
                            'attrs' => $listItem,
                            'detail' => $detailResult,
                            'dateline' => time(),
                        ];
                        $this->insertCrawlData('crawl_content_tmp', [
                            'tid' => $taskProperties['tid'],
                            'fid' => $taskProperties['fid'],
                            'url' => $detailUrl,
                            'content' => json_encode($crawlContentArray, JSON_UNESCAPED_UNICODE),
                            'dateline' => time(),
                        ]);
                        $crawlStat['availablecount'] = $crawlStat['availablecount'] + 1;
                        $this->setUrlVisited($detailUrl, $taskProperties['tid']);
                    } else {
                        $crawlStat['errordetails'][] = $detailUrl;
                    }
                    $this->sleepAlittle();
                }
            }
        }

        $crawlStat['endtime'] = time();

        $this->insertCrawlData('crawl_log', [
            'tid' => $taskProperties['tid'],
            'info' => json_encode($crawlStat, JSON_UNESCAPED_UNICODE),
            'dateline' => time(),
        ]);

        $this->insertCrawlDataIntoPost($taskAction);
    }

    /**
     * @param $url string
     * @return array
     */
    private function doCrawlDetail($url)
    {
        $detailResult = array();
        $crawlHelper = $this->getCrawlHelper();
        $detailUrl = $url;
        while(true) {
            $hasMore = false;
            $this->sleepAlittle();
            $respData = $crawlHelper->loadDetail($detailUrl);
            if(!empty($respData)) {
                $detailArray = @$respData['detail'];
                $npurl = @$respData['npurl'];
                if(!empty($detailArray)) {
                    if(!empty($detailResult)) {
                        foreach($detailResult as $key => $value) {
                            if(is_array($detailResult[$key])) {
                                if(!empty($detailArray[$key])) {
                                    $detailResult[$key] = array_merge($detailResult[$key], $detailArray[$key]);
                                }
                            }
                        }
                    } else {
                        $detailResult = $detailArray;
                    }
                    if(!empty($npurl)) {
                        $hasMore = true;
                        $detailUrl = $npurl;
                    }
                }
            }
            if(!$hasMore) {
                break;
            }
        }
        return $detailResult;
    }

    public function sleepAlittle()
    {
        $seconds = rand(1,2);
        sleep($seconds);
    }

    /**
     * @param $url string
     * @return bool
     */
    public function isUrlVisited($url)
    {
        try {
            $dataQuery = (new Query())->select('umd')
                ->from('crawl_url_visited')
                ->where('umd=:umd', [
                    ':umd' => md5($url),
                ])
                ->createCommand(Yii::$app->db)
                ->queryOne();
            if(!empty($dataQuery)) {
                return true;
            }
        } catch(\Exception $e) {
        }
        return false;
    }

    /**
     * @param $url string
     * @param $tid string
     */
    public function setUrlVisited($url, $tid)
    {
        try {
            $dbConn = Yii::$app->db;
            $dateline = time();
            $dbConn->createCommand()
                ->insert('crawl_url_visited', [
                    'umd' => md5($url),
                    'tid' => $tid,
                    'createat' => $dateline,
                    'updateat' => $dateline,
                ])
                ->execute();
        } catch(\Exception $e) {
        }
    }

    /**
     * @param $taskAction string
     * @return array|false
     */
    private function getTaskPropertiesByAction($taskAction)
    {
        $propertyArray = array();
        try {
            $dbConn = Yii::$app->db;
            $dataQuery = (new Query())->select('t.*, a.uid')
                ->from('crawl_task as t')
                ->innerJoin('crawl_entity as a',
                    't.eid=a.eid')
                ->where('action=:action', [
                    ':action' => $taskAction,
                ])
                ->createCommand($dbConn)
                ->queryOne();
            if(!empty($dataQuery)) {
                $propertyArray = $dataQuery;
            }
        } catch(\Exception $e) {
        }
        return $propertyArray;
    }

    /**
     * @param $tableName string
     * @param $insertData array
     */
    private function insertCrawlData($tableName, $insertData)
    {
        try {
            if(!empty($insertData)) {
                $dbConn = Yii::$app->db;

                $dbConn->createCommand()
                    ->insert($tableName, $insertData)
                    ->execute();
            }
        } catch(\Exception $e) {
        }
    }

    private function insertCrawlDataIntoPost($taskAction)
    {
        $taskProperties = $this->getTaskPropertiesByAction($taskAction);
        if(empty($taskProperties)) {
            return;
        }
        $dbConn = Yii::$app->db;
        $tableNameCrawlcontenttmp = 'crawl_content_tmp';
        $tableNameCrawlcontent = 'crawl_content';
        $dbConn->createCommand()
            ->update($tableNameCrawlcontenttmp, [
                    'status' => '1',
                ],
                'tid=:tid and fid=:fid and status=:status', [
                    ':tid' => $taskProperties['tid'],
                    ':fid' => $taskProperties['fid'],
                    ':status' => '0',
                ])
            ->execute();

        $dataQuery = (new Query())->select('id,content')
            ->from($tableNameCrawlcontenttmp)
            ->where('tid=:tid and fid=:fid and status=:status', [
                ':tid' => $taskProperties['tid'],
                ':fid' => $taskProperties['fid'],
                ':status' => '1',
            ])
            ->createCommand($dbConn)
            ->queryAll();

        if(!empty($dataQuery)) {
            foreach($dataQuery as $item) {
                if(!empty($item)) {
                    $itemData = json_decode($item['content'], true);
                    $postData = array();
                    $this->convertCrawlDataToPost($itemData, $taskAction, $postData);
                    $postObj = new ForumPost();
                    $postObj->fromArray($postData);
                    $insertResult = $postObj->insertPost();
                    if(!empty($insertResult)) {
                        $dbConn->createCommand()
                            ->update($tableNameCrawlcontenttmp, [
                                    'status' => '2',
                                    'ftid' => $insertResult['pid'],
                                ],
                                'id=:id', [
                                    ':id' => $item['id'],
                                ])
                            ->execute();
                    }
                }
            }
        }

        $sqlInsert = <<<EOD
INSERT INTO {$tableNameCrawlcontent}(`tid`,`fid`,`ftid`,`url`,`content`,`dateline`) 
    SELECT `tid`,`fid`,`ftid`,`url`,`content`,`dateline` FROM {$tableNameCrawlcontenttmp} WHERE tid=:tid AND fid=:fid AND `status`=:status;
EOD;
        $dbConn->createCommand($sqlInsert, [
                ':tid' => $taskProperties['tid'],
                ':fid' => $taskProperties['fid'],
                ':status' => '2',
            ])
            ->execute();

        $dbConn->createCommand()
            ->delete($tableNameCrawlcontenttmp,
                'tid=:tid and fid=:fid', [
                    ':tid' => $taskProperties['tid'],
                    ':fid' => $taskProperties['fid'],
                ])
            ->execute();
    }

    /**
     * @param $crawlData array
     * @param $taskAction string
     */
    public function convertCrawlDataToPost($crawlData, $taskAction, &$postData)
    {
        $taskProperties = $this->getTaskPropertiesByAction($taskAction);
        if(!empty($taskProperties)) {
            $postData['fid'] = $taskProperties['fid'];
            $postData['authorid'] = $taskProperties['uid'];
            $postData['author'] = $taskProperties['name'];
        }
    }

    /**
     * @return CrawlHelper
     */
    public function getCrawlHelper()
    {
        if($this->crawlHelper == null) {
            $this->crawlHelper = new CrawlHelper();
        }
        return $this->crawlHelper;
    }

    /**
     * @param $rulePath string relative path
     * @return bool|string
     */
    public function getRuleContent($rulePath)
    {
        $commonPath = Yii::getAlias("@common");
        $filePath = str_replace('/', DIRECTORY_SEPARATOR, $rulePath);
        $filePath =  "$commonPath" . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . $filePath;
        $ruleData = file_get_contents($filePath, false, null);
        return $ruleData;
    }
}

